// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
using System.Collections.Concurrent;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Azure.Identity;
using Azure.Monitor.OpenTelemetry.Exporter;
using Microsoft.ApplicationInsights.AspNetCore;
using Microsoft.ApplicationInsights.AspNetCore.Extensions;
using Microsoft.ApplicationInsights.DependencyCollector;
using Microsoft.ApplicationInsights.Extensibility;
using Microsoft.ApplicationInsights.Extensibility.Implementation.ApplicationId;
using Microsoft.Azure.WebJobs.Host;
using Microsoft.Azure.WebJobs.Logging;
using Microsoft.Azure.WebJobs.Logging.ApplicationInsights;
using Microsoft.Azure.WebJobs.Script.Config;
using Microsoft.Azure.WebJobs.Script.Diagnostics;
using Microsoft.Azure.WebJobs.Script.Eventing;
using Microsoft.Azure.WebJobs.Script.Metrics;
using Microsoft.Azure.WebJobs.Script.Scale;
using Microsoft.Azure.WebJobs.Script.WebHost.Configuration;
using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics;
using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics.Extensions;
using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using OpenTelemetry.Logs;
using OpenTelemetry.Metrics;
using OpenTelemetry.Trace;
using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames;
using AppInsightsCredentialOptions = Microsoft.Azure.WebJobs.Logging.ApplicationInsights.TokenCredentialOptions;
using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime;

namespace Microsoft.Azure.WebJobs.Script.WebHost
{
    public class WebJobsScriptHostService : IHostedService, IScriptHostManager, IServiceProvider, IDisposable
    {
        private readonly IApplicationLifetime _applicationLifetime;
        private readonly IOptionsMonitor<ScriptApplicationHostOptions> _applicationHostOptions;
        private readonly IScriptWebHostEnvironment _scriptWebHostEnvironment;
        private readonly IScriptHostBuilder _scriptHostBuilder;
        private readonly ILogger _logger;
        private readonly IEnvironment _environment;
        private readonly IMetricsLogger _metricsLogger;
        private readonly HostPerformanceManager _performanceManager;
        private readonly IOptions<HostHealthMonitorOptions> _healthMonitorOptions;
        private readonly IConfiguration _config;
        private readonly SlidingWindow<bool> _healthCheckWindow;
        private readonly Timer _hostHealthCheckTimer;
        private readonly SemaphoreSlim _hostStartSemaphore = new SemaphoreSlim(1, 1);
        private readonly TaskCompletionSource<bool> _hostStartedSource = new TaskCompletionSource<bool>();
        private readonly Task _hostStarted;
        private readonly IHostMetrics _hostMetrics;
        private readonly IOptions<FunctionsHostingConfigOptions> _hostingConfigOptions;
        private readonly bool _originalStandbyModeValue;
        private readonly string _originalFunctionsWorkerRuntime;
        private readonly string _originalFunctionsWorkerRuntimeVersion;
        private readonly IOptionsChangeTokenSource<WorkerConfigurationResolverOptions> _workerConfigResolverOptionsChangeTokenSource;
        private readonly IOptionsChangeTokenSource<LanguageWorkerOptions> _languageWorkerOptionsChangeTokenSource;

        // we're only using this dictionary's keys so it acts as a "ConcurrentHashSet"
        private readonly ConcurrentDictionary<ScriptHostStartupOperation, byte> _activeStartupOperations = new();

        private IScriptEventManager _eventManager;
        private IHost _host;
        private ScriptHostState _state;
        private CancellationTokenSource _startupLoopTokenSource;
        private int _hostStartCount;
        private bool _disposed = false;

        private static IDisposable _telemetryConfiguration;
        private static IDisposable _requestTrackingModule;

        private int _applicationStopping;
        private int _applicationStopped;

        public WebJobsScriptHostService(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, IScriptHostBuilder scriptHostBuilder, ILoggerFactory loggerFactory,
            IScriptWebHostEnvironment scriptWebHostEnvironment, IEnvironment environment,
            HostPerformanceManager hostPerformanceManager, IOptions<HostHealthMonitorOptions> healthMonitorOptions,
            IMetricsLogger metricsLogger, IApplicationLifetime applicationLifetime, IConfiguration config, IScriptEventManager eventManager, IHostMetrics hostMetrics,
            IOptions<FunctionsHostingConfigOptions> hostingConfigOptions,
            IOptionsChangeTokenSource<LanguageWorkerOptions> languageWorkerOptionsChangeTokenSource,
            IOptionsChangeTokenSource<WorkerConfigurationResolverOptions> workerConfigResolverOptionsChangeTokenSource)
        {
            ArgumentNullException.ThrowIfNull(loggerFactory);

            // This will no-op if already initialized.
            InitializeApplicationInsightsRequestTracking();

            _applicationLifetime = applicationLifetime;
            RegisterApplicationLifetimeEvents();

            _metricsLogger = metricsLogger;
            _workerConfigResolverOptionsChangeTokenSource = workerConfigResolverOptionsChangeTokenSource ?? throw new ArgumentNullException(nameof(workerConfigResolverOptionsChangeTokenSource));
            _languageWorkerOptionsChangeTokenSource = languageWorkerOptionsChangeTokenSource ?? throw new ArgumentNullException(nameof(languageWorkerOptionsChangeTokenSource));
            _applicationHostOptions = applicationHostOptions ?? throw new ArgumentNullException(nameof(applicationHostOptions));
            _scriptWebHostEnvironment = scriptWebHostEnvironment ?? throw new ArgumentNullException(nameof(scriptWebHostEnvironment));
            _scriptHostBuilder = scriptHostBuilder ?? throw new ArgumentNullException(nameof(scriptHostBuilder));
            _environment = environment ?? throw new ArgumentNullException(nameof(environment));
            _performanceManager = hostPerformanceManager ?? throw new ArgumentNullException(nameof(hostPerformanceManager));
            _healthMonitorOptions = healthMonitorOptions ?? throw new ArgumentNullException(nameof(healthMonitorOptions));
            _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
            _config = config ?? throw new ArgumentNullException(nameof(config));
            _eventManager = eventManager;
            _hostMetrics = hostMetrics ?? throw new ArgumentNullException(nameof(hostMetrics));

            _hostStarted = _hostStartedSource.Task;

            State = ScriptHostState.Default;

            if (ShouldMonitorHostHealth)
            {
                _healthCheckWindow = new SlidingWindow<bool>(_healthMonitorOptions.Value.HealthCheckWindow);
                _hostHealthCheckTimer = new Timer(OnHostHealthCheckTimer, null, TimeSpan.Zero, _healthMonitorOptions.Value.HealthCheckInterval);
            }

            _hostingConfigOptions = hostingConfigOptions;

            // we'll use this to emit telemetry on if and how this process has been specialized
            _originalStandbyModeValue = _scriptWebHostEnvironment.InStandbyMode;
            _originalFunctionsWorkerRuntime = _environment.GetFunctionsWorkerRuntime();
            _originalFunctionsWorkerRuntimeVersion = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName);
        }

        public event EventHandler HostInitializing;

        public event EventHandler<ActiveHostChangedEventArgs> ActiveHostChanged;

        [Flags]
        private enum JobHostStartupMode
        {
            Normal = 0,
            Offline = 1,
            HandlingError = 2,
            HandlingNonTransientError = 4 | HandlingError,
            HandlingConfigurationParsingError = 8 | HandlingError,
            HandlingInitializationError = 16 | HandlingNonTransientError
        }

        private bool ShutdownRequested => _applicationStopping == 1 || _applicationStopped == 1;

        private IHost ActiveHost
        {
            get
            {
                return _host;
            }

            set
            {
                _logger.ActiveHostChanging(GetHostInstanceId(_host), GetHostInstanceId(value));

                var previousHost = _host;
                _host = value;

                OnActiveHostChanged(previousHost, _host);
            }
        }

        public IServiceProvider Services => ActiveHost?.Services;

        public ScriptHostState State
        {
            get
            {
                return _state;
            }

            private set
            {
                if (_state != value)
                {
                    _logger.HostStateChanged(_state, value);
                }

                _state = value;
            }
        }

        public Exception LastError { get; private set; }

        /// <summary>
        /// Gets a value indicating whether the host health monitor should be active.
        /// </summary>
        internal bool ShouldMonitorHostHealth
        {
            get
            {
                return _healthMonitorOptions.Value.Enabled && _environment.IsAppService() && !_scriptWebHostEnvironment.InStandbyMode;
            }
        }

        public async Task StartAsync(CancellationToken cancellationToken)
        {
            if (ShutdownRequested)
            {
                return;
            }

            _startupLoopTokenSource = new CancellationTokenSource();
            var startupLoopToken = _startupLoopTokenSource.Token;
            var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(startupLoopToken, cancellationToken);

            try
            {
                await StartHostAsync(tokenSource.Token);
            }
            catch (OperationCanceledException ex)
            {
                _logger.LogWarning("Host startup operation has been canceled", ex);

                if (cancellationToken.IsCancellationRequested)
                {
                    _logger.ScriptHostServiceInitCanceledByRuntime();
                    throw;
                }

                // If the exception was triggered by our loop cancellation token, just ignore as
                // it doesn't indicate an issue.
            }
        }

        private async Task CheckFileSystemAsync()
        {
            if (_environment.ZipDeploymentAppSettingsExist())
            {
                // Check for marker file indicating a zip package failure, and if found stop the application.
                // We never want to run with an incorrect file system.
                string path = Path.Combine(_applicationHostOptions.CurrentValue.ScriptPath, ScriptConstants.RunFromPackageFailedFileName);

                if (File.Exists(path))
                {
                    string errorPrefix = $"Shutting down host due to presence of {path}.";
                    string errorSuffix = string.Empty;
                    try
                    {
                        string fileContent = File.ReadAllText(path);
                        errorSuffix = $" File content: {fileContent}";
                    }
                    catch (Exception ex)
                    {
                        errorSuffix = $" Error reading {ScriptConstants.RunFromPackageFailedFileName} file content: {ex.Message}";
                    }
                    finally
                    {
                        var errorMessage = $"{errorPrefix}{errorSuffix}";

                        _logger.LogError(errorMessage);
                        await LogErrorWithTransientOtelLoggerAsync(errorMessage);
                    }
                    _applicationLifetime.StopApplication();
                }
            }
        }

        private void ValidateLinuxSKUConfiguration(ILogger logger)
        {
            var websiteRunFromPackageValue = _environment.GetEnvironmentVariable(AzureWebsiteRunFromPackage);
            var scmRunFromPackageValue = _environment.GetEnvironmentVariable(ScmRunFromPackage);

            if (string.IsNullOrEmpty(websiteRunFromPackageValue) &&
                string.IsNullOrEmpty(scmRunFromPackageValue) &&
                _environment.IsLinuxConsumptionOnAtlas() &&
                !_environment.IsManagedAppEnvironment())
            {
                logger?.LogError($"Unable to load the functions payload since the app was not provisioned with valid {AzureWebJobsSecretStorage} connection string.");
            }

            var functionsTimeZone = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsTimeZone);
            var functionsWebsiteTimeZone = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsWebsiteTimeZone);

            // If we have a linux consumption app and the time zone env variable is set, log a warning and diagnostic event
            if (_environment.IsLinuxConsumptionOnAtlas() &&
                (!string.IsNullOrEmpty(functionsTimeZone) ||
                !string.IsNullOrEmpty(functionsWebsiteTimeZone)))
            {
                string message = Script.Properties.Resources.LinuxConsumptionRemoveTimeZone;

                // Log diagnostic event
                logger?.LogDiagnosticEventError(DiagnosticEventConstants.LinuxConsumptionTimeZoneErrorCode, message, DiagnosticEventConstants.LinuxConsumptionTimeZoneErrorHelpLink, new InvalidOperationException(message));

                // Log warning so this message goes to App insights
                logger?.LogWarning(message);
            }
        }

        private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0,
            JobHostStartupMode startupMode = JobHostStartupMode.Normal, Guid? parentOperationId = null)
        {
            // Add this to the list of trackable startup operations. Restarts can use this to cancel any ongoing or pending operations.
            var activeOperation = BeginStartupOperation(cancellationToken, parentOperationId);

            using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartService))
            {
                try
                {
                    await _hostStartSemaphore.WaitAsync();

                    // Now that we're inside the semaphore, set this task as completed. This prevents
                    // restarts from being invoked (via the PlaceholderSpecializationMiddleware) before
                    // the IHostedService has ever started.
                    _hostStartedSource.TrySetResult(true);

                    await UnsynchronizedStartHostAsync(activeOperation, attemptCount, startupMode);
                }
                finally
                {
                    EndStartupOperation(activeOperation);
                    _hostStartSemaphore.Release();
                }
            }
        }

        /// <summary>
        /// Starts the host without taking a lock. Callers must take a lock on _hostStartSemaphore
        /// before calling this method. Host starts and restarts must be synchronous to prevent orphaned
        /// hosts or an incorrect ActiveHost.
        /// </summary>
        private Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal)
        {
            // We may be started from a variety of contexts, some of which may carry AsyncLocal state (such as Activity). We do not want any of this
            // to flow into the host startup logic, so we suppress the flow of the ExecutionContext.
            Task start;
            using (System.Threading.ExecutionContext.SuppressFlow())
            {
                start = UnsynchronizedStartHostCoreAsync(activeOperation, attemptCount, startupMode);
            }

            return start;
        }

        /// <summary>
        /// Starts the host without taking a lock. Callers must take a lock on _hostStartSemaphore
        /// before calling this method. Host starts and restarts must be synchronous to prevent orphaned
        /// hosts or an incorrect ActiveHost.
        /// </summary>
        private async Task UnsynchronizedStartHostCoreAsync(ScriptHostStartupOperation activeOperation, int attemptCount, JobHostStartupMode startupMode)
        {
            await Task.Yield(); // ensure any async context is properly suppressed.
            await CheckFileSystemAsync();
            if (ShutdownRequested)
            {
                return;
            }

            IHost localHost = null;
            var currentCancellationToken = activeOperation.CancellationTokenSource.Token;
            _logger.StartupOperationStarting(activeOperation.Id);

            try
            {
                currentCancellationToken.ThrowIfCancellationRequested();

                // if we were in an error state retain that,
                // otherwise move to default
                if (State != ScriptHostState.Error)
                {
                    State = ScriptHostState.Default;
                }

                bool isOffline = Utility.CheckAppOffline(_applicationHostOptions.CurrentValue.ScriptPath);
                State = isOffline ? ScriptHostState.Offline : State;
                bool hasNonTransientErrors = startupMode.HasFlag(JobHostStartupMode.HandlingNonTransientError);
                bool handlingError = startupMode.HasFlag(JobHostStartupMode.HandlingError);

                // If we're in a non-transient error state or offline, skip host initialization
                bool skipJobHostStartup = isOffline || hasNonTransientErrors;
                bool skipHostJsonConfiguration = startupMode == JobHostStartupMode.HandlingConfigurationParsingError;
                string functionsExtensionVersion = _environment.GetFunctionsExtensionVersion();
                _logger.Building(functionsExtensionVersion, skipJobHostStartup, skipHostJsonConfiguration, activeOperation.Id);

                using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerBuildScriptHost))
                {
                    localHost = BuildHost(skipJobHostStartup, skipHostJsonConfiguration);
                }

                ActiveHost = localHost;

                if (!FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableWebHostLogForwarding, _environment))
                {
                    // Forward logs to AppInsights/OpenTelemetry.
                    // These are not tracked by the AppInsights and OpenTelemetry logger provider as these are added in the script host.
                    var loggerProviders = ActiveHost.Services.GetServices<ILoggerProvider>();
                    var deferredLogProvider = ActiveHost.Services.GetService<DeferredLoggerProvider>();
                    if (deferredLogProvider is not null)
                    {
                        var selectedProviders = loggerProviders.Where(provider => provider is ApplicationInsightsLoggerProvider or OpenTelemetryLoggerProvider).ToArray();
                        _ = Task.Run(() => deferredLogProvider.ProcessBufferedLogsAsync(selectedProviders));
                    }
                }

                if (_workerConfigResolverOptionsChangeTokenSource is HostBuiltChangeTokenSource<WorkerConfigurationResolverOptions> { } hostBuiltChangeTokenResolverOptions)
                {
                    hostBuiltChangeTokenResolverOptions.TriggerChange();
                }

                if (_languageWorkerOptionsChangeTokenSource is HostBuiltChangeTokenSource<LanguageWorkerOptions> { } hostBuiltChangeTokenSource)
                {
                    hostBuiltChangeTokenSource.TriggerChange();
                }

                var scriptHost = (ScriptHost)ActiveHost.Services.GetService<ScriptHost>();
                if (scriptHost != null)
                {
                    scriptHost.HostInitializing += OnHostInitializing;

                    if (!handlingError)
                    {
                        // Services may be initialized, but we don't want set the state to Initialized as we're
                        // handling an error and want to retain the Error state.
                        scriptHost.HostInitialized += OnHostInitialized;
                    }
                }

                LogInitialization(localHost, isOffline, attemptCount, ++_hostStartCount, activeOperation.Id);

                ValidateLinuxSKUConfiguration(GetHostLogger(localHost));

                if (!_scriptWebHostEnvironment.InStandbyMode)
                {
                    // At this point we know that App Insights is initialized (if being used), so we
                    // can dispose this early request tracking module, which forces our new one to take over.
                    DisposeRequestTrackingModule();
                }

                currentCancellationToken.ThrowIfCancellationRequested();

                var hostInstanceId = GetHostInstanceId(localHost);
                _logger.StartupOperationStartingHost(activeOperation.Id, hostInstanceId);

                using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartScriptHost))
                {
                    await localHost.StartAsync(currentCancellationToken);
                }

                if (!handlingError)
                {
                    LastError = null;

                    if (!isOffline)
                    {
                        State = ScriptHostState.Running;
                    }
                }

                _eventManager.Publish(new HostStartEvent());
            }
            catch (OperationCanceledException)
            {
                GetHostLogger(localHost).StartupOperationWasCanceled(activeOperation.Id);
                throw;
            }
            catch (Exception exc)
            {
                bool isActiveHost = ReferenceEquals(localHost, ActiveHost);
                ILogger logger = GetHostLogger(localHost);

                if (isActiveHost)
                {
                    LastError = exc;
                    State = ScriptHostState.Error;
                    logger.ErrorOccurredDuringStartupOperation(activeOperation.Id, exc);
                }
                else
                {
                    // Another host has been created before this host
                    // threw its startup exception. We want to make sure it
                    // doesn't control the state of the service.
                    logger.ErrorOccurredInactive(activeOperation.Id, exc);
                }

                attemptCount++;

                if (attemptCount > 3)
                {
                    _hostMetrics.AppFailure();
                }

                if (ShutdownHostIfUnhealthy())
                {
                    return;
                }

                if (isActiveHost)
                {
                    // We don't want to return disposed services via the Services property, so
                    // set this to null before calling Orphan().
                    ActiveHost = null;
                }

                var orphanTask = Orphan(localHost)
                    .ContinueWith(t =>
                    {
                        if (t.IsFaulted)
                        {
                            t.Exception.Handle(e => true);
                        }
                    }, TaskContinuationOptions.ExecuteSynchronously);

                // Use the fallback logger now, as we cannot trust when the host
                // logger will be disposed.
                logger = _logger;

                if (currentCancellationToken.IsCancellationRequested)
                {
                    logger.CancellationRequested(activeOperation.Id);
                    currentCancellationToken.ThrowIfCancellationRequested();
                }

                var nextStartupAttemptMode = JobHostStartupMode.Normal;

                if (exc is HostConfigurationException)
                {
                    // Try starting the host without parsing host.json. This will start up a
                    // minimal host and allow the portal to see the error. Any modification will restart again.
                    nextStartupAttemptMode = JobHostStartupMode.HandlingConfigurationParsingError;
                }
                else if (exc is HostInitializationException)
                {
                    nextStartupAttemptMode = JobHostStartupMode.HandlingInitializationError;
                }

                if (nextStartupAttemptMode != JobHostStartupMode.Normal)
                {
                    logger.LogDebug($"Starting new host with '{nextStartupAttemptMode}' and parent operation id '{activeOperation.Id}'.");
                    Task ignore = StartHostAsync(currentCancellationToken, attemptCount, nextStartupAttemptMode, activeOperation.Id);
                }
                else
                {
                    logger.LogDebug("Will start a new host after delay.");

                    await Utility.DelayWithBackoffAsync(attemptCount, currentCancellationToken, min: TimeSpan.FromSeconds(1), max: TimeSpan.FromMinutes(2), logger: logger);

                    if (currentCancellationToken.IsCancellationRequested)
                    {
                        logger.LogDebug($"Cancellation for operation '{activeOperation.Id}' requested during delay. A new host will not be started.");
                        currentCancellationToken.ThrowIfCancellationRequested();
                    }

                    logger.LogDebug("Starting new host after delay.");
                    Task ignore = StartHostAsync(currentCancellationToken, attemptCount, parentOperationId: activeOperation.Id);
                }
            }
        }

        private void DisposeRequestTrackingModule()
        {
            _requestTrackingModule?.Dispose();
            _requestTrackingModule = null;

            _telemetryConfiguration?.Dispose();
            _telemetryConfiguration = null;
        }

        public async Task StopAsync(CancellationToken cancellationToken)
        {
            _startupLoopTokenSource?.Cancel();

            State = ScriptHostState.Stopping;
            _logger.Stopping();

            var currentHost = ActiveHost;
            ActiveHost = null;
            Task stopTask = Orphan(currentHost, cancellationToken);
            Task result = await Task.WhenAny(stopTask, Task.Delay(TimeSpan.FromSeconds(10)));

            if (result != stopTask)
            {
                _logger.DidNotShutDown();
            }
            else
            {
                _logger.ShutDownCompleted();
            }

            State = ScriptHostState.Stopped;
        }

        public async Task RestartHostAsync(string reason, CancellationToken cancellationToken)
        {
            ArgumentException.ThrowIfNullOrWhiteSpace(reason);

            if (ShutdownRequested)
            {
                return;
            }

            using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerRestartService))
            {
                // Do not invoke a restart if the host has not yet been started. This can lead
                // to invalid state.
                if (!_hostStarted.IsCompleted)
                {
                    _logger.RestartBeforeStart();
                    await _hostStarted;
                }

                _logger.EnteringRestart(reason);

                // If anything is mid-startup, cancel it.
                _startupLoopTokenSource?.Cancel();
                foreach (var startupOperation in _activeStartupOperations.Keys)
                {
                    _logger.CancelingStartupOperationForRestart(startupOperation.Id);
                    try
                    {
                        startupOperation.CancellationTokenSource.Cancel();
                    }
                    catch (ObjectDisposedException)
                    {
                        // This can be disposed at any time.
                    }
                }

                try
                {
                    await _hostStartSemaphore.WaitAsync();

                    if (State == ScriptHostState.Stopping || State == ScriptHostState.Stopped)
                    {
                        _logger.SkipRestart(State.ToString());
                        return;
                    }

                    State = ScriptHostState.Default;
                    _logger.Restarting();

                    var previousHost = ActiveHost;
                    ActiveHost = null;

                    var activeOperation = BeginStartupOperation(cancellationToken);

                    try
                    {
                        Task startTask, stopTask;

                        // If we are running in development mode with core tools, do not overlap the restarts.
                        // Overlapping restarts are problematic when language worker processes are listening
                        // to the same debug port
                        if (ShouldEnforceSequentialRestart(previousHost))
                        {
                            stopTask = Orphan(previousHost, cancellationToken);
                            await stopTask;
                            startTask = UnsynchronizedStartHostAsync(activeOperation);
                        }
                        else
                        {
                            await NotifyHostStoppingAsync(previousHost, cancellationToken);
                            startTask = UnsynchronizedStartHostAsync(activeOperation);
                            stopTask = Orphan(previousHost, cancellationToken);
                        }

                        await startTask;
                    }
                    finally
                    {
                        EndStartupOperation(activeOperation);
                    }

                    _logger.Restarted();
                }
                catch (OperationCanceledException)
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        _logger.ScriptHostServiceRestartCanceledByRuntime();
                        throw;
                    }

                    // If the exception was triggered by our startup operation cancellation token, just ignore as
                    // it doesn't indicate an issue.
                }
                finally
                {
                    _hostStartSemaphore.Release();
                }
            }
        }

        // Because we fire-and-forget the host disposal, we cannot be guaranteed when it will be stopped
        // or disposed. Use this method to explicitly stop any services in the host that may be
        // problematic to run side-by-side with the new host that is starting.
        private static Task NotifyHostStoppingAsync(IHost previousHost, CancellationToken cancellationToken)
        {
            var scriptHost = previousHost?.Services?.GetService<ScriptHost>();
            return scriptHost?.NotifyStoppingAsync(cancellationToken) ?? Task.CompletedTask;
        }

        internal bool ShouldEnforceSequentialRestart(IHost host = null)
        {
            var options = host?.Services?.GetService<IOptions<ScriptHostRecycleOptions>>().Value;
            options ??= ScriptHostRecycleOptions.Create(_config);
            return options.SequentialHostRestartRequired;
        }

        private void OnHostInitializing(object sender, EventArgs e)
        {
            // we check host health before starting to avoid starting
            // the host when connection or other issues exist
            IsHostHealthy(throwWhenUnhealthy: true);

            // We invoke any registered event delegates during Host Initialization
            HostInitializing?.Invoke(sender, e);
        }

        private void OnActiveHostChanged(IHost previousHost, IHost newHost)
        {
            ActiveHostChanged?.Invoke(this, new ActiveHostChangedEventArgs(previousHost, _host));
        }

        /// <summary>
        /// Called after the host has been fully initialized, but before it
        /// has been started.
        /// </summary>
        private void OnHostInitialized(object sender, EventArgs e)
        {
            State = ScriptHostState.Initialized;
        }

        private IHost BuildHost(bool skipHostStartup, bool skipHostJsonConfiguration)
        {
            return _scriptHostBuilder.BuildHost(skipHostStartup, skipHostJsonConfiguration);
        }

        private string GetHostInstanceId(IHost host)
        {
            IOptions<ScriptJobHostOptions> scriptHostOptions = null;

            try
            {
                scriptHostOptions = host?.Services.GetService<IOptions<ScriptJobHostOptions>>();
            }
            catch (ObjectDisposedException)
            {
                // If the host is disposed, we cannot access services.
            }

            return scriptHostOptions?.Value?.InstanceId;
        }

        private ILogger GetHostLogger(IHost host)
        {
            ILoggerFactory hostLoggerFactory = null;
            try
            {
                hostLoggerFactory = host?.Services?.GetService<ILoggerFactory>();
            }
            catch (ObjectDisposedException)
            {
                // If the host is disposed, we cannot access services.
            }

            // Attempt to get the host logger with JobHost configuration applied
            // using the default logger as a fallback
            if (hostLoggerFactory is not null)
            {
                return hostLoggerFactory?.CreateLogger(LogCategories.Startup);
            }

            // An error occurred before the host was built; a minimal logger factory is being created to send telemetry to AppInsights/Otel.
            var otelLoggerFactory = BuildOtelLoggerFactory();

            // If the Otel logger factory is null, use the fallback logger instead. These logs will not be accessible in AppInsights/Otel.
            if (otelLoggerFactory is null)
            {
                return _logger;
            }
            return new CompositeLogger(_logger, otelLoggerFactory.CreateLogger(LogCategories.Startup));
        }

        private void LogInitialization(IHost host, bool isOffline, int attemptCount, int startCount, Guid operationId)
        {
            var logger = GetHostLogger(host);

            if (isOffline)
            {
                logger.Offline(operationId);
            }
            else
            {
                logger.Initializing(operationId);
            }
            logger.Initialization(attemptCount, startCount, operationId);

            if (_scriptWebHostEnvironment.InStandbyMode)
            {
                // Reading the string from resources to make sure resource loading code path is warmed up during placeholder as well.
                logger.InStandByMode(operationId);
            }

            // Log settings
            var functionWorkerRuntime = _environment.GetEnvironmentVariable(FunctionWorkerRuntime);
            var functionWorkerRuntimeVersion = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName);
            var functionExtensionVersion = _environment.GetEnvironmentVariable(FunctionsExtensionVersion);
            var currentDirectory = Directory.GetCurrentDirectory();
            var inStandbyMode = _scriptWebHostEnvironment.InStandbyMode;
            var hasBeenSpecialized = _originalStandbyModeValue && !inStandbyMode;
            var usePlaceholderDotNetIsolated = _environment.UsePlaceholderDotNetIsolated();
            var websiteSku = _environment.GetEnvironmentVariable(AzureWebsiteSku);
            var featureFlags = _environment.GetEnvironmentVariable(AzureWebJobsFeatureFlags);
            var hostingConfigDict = _hostingConfigOptions.Value.Features;

            string hisMode = "Disabled";
            if (FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagStrictHISModeEnabled))
            {
                hisMode = "Strict";
                _metricsLogger.LogEvent(MetricEventNames.HISStrictModeEnabled);
                _logger.LogDebug($"HIS Strict mode enabled.");
            }
            else if (FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagStrictHISModeWarn))
            {
                hisMode = "Warn";
                _metricsLogger.LogEvent(MetricEventNames.HISStrictModeWarn);
                _logger.LogDebug($"HIS Warn mode enabled.");
            }

            logger.LogHostInitializationSettings(_originalFunctionsWorkerRuntime, functionWorkerRuntime, _originalFunctionsWorkerRuntimeVersion, functionWorkerRuntimeVersion,
                functionExtensionVersion, currentDirectory, inStandbyMode, hasBeenSpecialized, usePlaceholderDotNetIsolated, websiteSku, featureFlags, hostingConfigDict, hisMode);
        }

        private void OnHostHealthCheckTimer(object state)
        {
            bool isHealthy = IsHostHealthy();
            _healthCheckWindow.AddEvent(isHealthy);

            if (!isHealthy && State == ScriptHostState.Running)
            {
                // This periodic check allows us to break out of the host run
                // loop. The health check performed in OnHostInitializing will then
                // fail and we'll enter a restart loop (exponentially backing off)
                // until the host is healthy again and we can resume host processing.
                _logger.UnhealthyRestart();
                var tIgnore = RestartHostAsync("Host health check detected an unhealthy host.", CancellationToken.None);
            }
        }

        internal bool IsHostHealthy(bool throwWhenUnhealthy = false)
        {
            if (!ShouldMonitorHostHealth)
            {
                return true;
            }

            var exceededCounters = new Collection<string>();
            if (_performanceManager.PerformanceCountersExceeded(exceededCounters))
            {
                string formattedCounters = string.Join(", ", exceededCounters);
                if (throwWhenUnhealthy)
                {
                    throw new InvalidOperationException($"Host thresholds exceeded: [{formattedCounters}]. For more information, see https://aka.ms/functions-thresholds.");
                }
                return false;
            }

            return true;
        }

        private bool ShutdownHostIfUnhealthy()
        {
            if (ShouldMonitorHostHealth && _healthCheckWindow.GetEvents().Count(isHealthy => !isHealthy) > _healthMonitorOptions.Value.HealthCheckThreshold)
            {
                // if the number of times the host has been unhealthy in
                // the current time window exceeds the threshold, recover by
                // initiating shutdown
                _logger.UnhealthyCountExceeded(_healthMonitorOptions.Value.HealthCheckThreshold, _healthMonitorOptions.Value.HealthCheckWindow);
                _applicationLifetime.StopApplication();
                return true;
            }

            return false;
        }

        /// <summary>
        /// Remove the <see cref="IHost"/> instance from the live instances collection,
        /// allowing it to finish currently executing functions before stopping and disposing of it.
        /// </summary>
        /// <param name="instance">The <see cref="IHost"/> instance to remove.</param>
        private async Task Orphan(IHost instance, CancellationToken cancellationToken = default)
        {
            if (instance == null)
            {
                _logger.LogDebug("Cannot stop or dispose a null host instance.");
                return;
            }

            var isStandbyHost = false;
            string hostInstanceId = "unknown";
            try
            {
                var scriptHost = instance.Services.GetService<ScriptHost>();
                if (scriptHost != null)
                {
                    scriptHost.HostInitializing -= OnHostInitializing;
                    scriptHost.HostInitialized -= OnHostInitialized;
                    isStandbyHost = scriptHost.IsStandbyHost;
                    hostInstanceId = scriptHost.ScriptOptions.InstanceId;
                }
            }
            catch (ObjectDisposedException)
            {
                // If the instance is already disposed, we cannot access its services.
            }

            try
            {
                await instance.StopAsync(cancellationToken);
            }
            catch (Exception ex) when (!ex.IsFatal())
            {
                // some errors are expected here - e.g. in error/shutdown situations
                // we might attempt to stop the host before it has fully started
                _logger.LogError(ex, "Failed to stop host instance '{hostInstanceId}'.", hostInstanceId);
            }
            finally
            {
                GetHostLogger(instance).LogDebug("Disposing ScriptHost.");

                if (isStandbyHost && !_scriptWebHostEnvironment.InStandbyMode)
                {
                    // For cold start reasons delay disposing script host if specializing out of placeholder mode
                    Utility.ExecuteAfterColdStartDelay(_environment, () =>
                    {
                        try
                        {
                            GetHostLogger(instance).LogDebug("Starting Standby ScriptHost dispose");
                            instance.Dispose();
                            _logger.LogDebug("Standby ScriptHost disposed");
                        }
                        catch (Exception e)
                        {
                            _logger.LogError(e, "Failed to dispose Standby ScriptHost instance");
                            throw;
                        }
                    }, cancellationToken);

                    _logger.LogDebug("Standby ScriptHost marked for disposal");
                }
                else
                {
                    DisposeDependencyTrackingModule(instance);
                    FlushOpenTelemetry(instance);
                    instance.Dispose();
                    _logger.LogDebug("ScriptHost disposed");
                }
            }
        }

        // A temporary fix until we are able to take App Insights 2.18.0. There is a potential
        // race during disposal where new DiagnosticListeners can throw exceptions after TelemetryConfiguration
        // is disposed, but before this module is disposed. This ensures the module disposes first.
        // Tracking issue: https://github.com/Azure/azure-functions-host/issues/7450
        private void DisposeDependencyTrackingModule(IHost instance)
        {
            try
            {
                var module = instance?.Services.GetServices<ITelemetryModule>()
                                      .SingleOrDefault(m => m is DependencyTrackingTelemetryModule)
                                      as IDisposable;

                module?.Dispose();

                _logger.LogDebug($"{nameof(DependencyTrackingTelemetryModule)} disposed.");
            }
            catch (Exception ex)
            {
                // best effort.
                _logger.LogDebug($"Unable to dispose {nameof(DependencyTrackingTelemetryModule)}. {ex}");
            }
        }

        private void RegisterApplicationLifetimeEvents()
        {
            _applicationLifetime.ApplicationStopping.Register(() =>
            {
                Interlocked.Exchange(ref _applicationStopping, 1);
                if (_environment.DrainOnApplicationStoppingEnabled())
                {
                    var drainModeManager = _host?.Services.GetService<IDrainModeManager>();
                    if (drainModeManager != null)
                    {
                        _logger.LogDebug("Application Stopping: initiate drain mode");
                        drainModeManager.EnableDrainModeAsync(CancellationToken.None);
                    }
                }
            });

            _applicationLifetime.ApplicationStopped.Register(() =>
            {
                Interlocked.Exchange(ref _applicationStopped, 1);
            });
        }

        private static void InitializeApplicationInsightsRequestTracking()
        {
            if (_requestTrackingModule != null)
            {
                return;
            }

            // Requests may come in before the JobHost has started (like during cold starts), which means
            // they will not be properly tracked by Application Insights because there's nothing listening
            // for them yet. This wires up the request tracking module with default values to catch those
            // events and properly create an Activity. Once the JobHost has started, we dispose this and the
            // JobHost tracking module takes over.
            var module = new RequestTrackingTelemetryModule(new ApplicationInsightsApplicationIdProvider())
            {
                CollectionOptions = new RequestCollectionOptions
                {
                    TrackExceptions = false,
                    EnableW3CDistributedTracing = true,
                    InjectResponseHeaders = true
                }
            };

            var telemetryConfig = new TelemetryConfiguration();

            module.Initialize(telemetryConfig);

            _telemetryConfiguration = telemetryConfig;
            _requestTrackingModule = module;
        }

        public object GetService(Type serviceType)
        {
            return Services?.GetService(serviceType);
        }

        private void FlushOpenTelemetry(IHost host)
        {
            var logger = GetHostLogger(host);
            foreach (var meterProvider in host.Services.GetServices<MeterProvider>())
            {
                logger.LogDebug(@"Flushing {providerName} ...", meterProvider);
                meterProvider.ForceFlush();
            }

            foreach (var tracerProvider in host.Services.GetServices<TracerProvider>())
            {
                logger.LogDebug(@"Flushing {providerName} ...", tracerProvider);
                tracerProvider.ForceFlush();
            }

            foreach (var logProvider in host.Services.GetServices<ILoggerProvider>().Where(i => i is OpenTelemetryLoggerProvider))
            {
                logger.LogDebug(@"Disposing {providerName} ...", logProvider);
                logProvider.Dispose();
            }
        }

        /// <summary>
        /// Creates a new startup operation and adds it to the list of active operations. This operation should be completed by
        /// calling <see cref="EndStartupOperation(ScriptHostStartupOperation)"/>."/>.
        /// </summary>
        /// <param name="parentToken">A CancellationToken to link to this operation. It can be canceled via the <see cref="ScriptHostStartupOperation.CancellationTokenSource"/> property.</param>
        /// <param name="parentId">The Id of the parent operation if this one is being created due to a startup exception.</param>
        /// <returns>The operation.</returns>
        private ScriptHostStartupOperation BeginStartupOperation(CancellationToken parentToken, Guid? parentId = null)
        {
            var operation = new ScriptHostStartupOperation(parentToken, parentId);
            _activeStartupOperations.TryAdd(operation, byte.MinValue);
            _logger.StartupOperationCreated(operation.Id, operation.ParentId);
            return operation;
        }

        /// <summary>
        /// Removes the startup operation from the list of active operations and disposes of it.
        /// </summary>
        /// <param name="operation">The operation to complete.</param>
        private void EndStartupOperation(ScriptHostStartupOperation operation)
        {
            if (_activeStartupOperations.TryRemove(operation, out _))
            {
                operation.Dispose();
            }

            _logger.StartupOperationCompleted(operation.Id);
        }

        private async Task LogErrorWithTransientOtelLoggerAsync(string log)
        {
            var loggerFactory = BuildOtelLoggerFactory();

            if (loggerFactory is not null)
            {
                var logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
                logger.LogError(log);

                // Delay increases the chance that the log is sent to AppInsights/Otel before the logger factory is disposed
                await Task.Delay(2000);
                // Do a force flush as the host is shutting down and we want to ensure the logs are sent before disposing the logger factory
                ForceFlush(loggerFactory);
                // Give some time for the logger to flush
                await Task.Delay(4000);
            }
        }

        private ILoggerFactory BuildOtelLoggerFactory()
        {
            var appInsightsConnStr = GetConfigurationValue(AppInsightsConnectionString, _config);
            var otlpEndpoint = GetConfigurationValue(OtlpEndpoint, _config);
            if (appInsightsConnStr is not { Length: > 0 } && otlpEndpoint is not { Length: > 0 })
            {
                return null; // Nothing configured
            }

            // Create a minimal logger factory with OpenTelemetry and Azure Monitor exporter
            return LoggerFactory.Create(builder =>
            {
                builder.AddOpenTelemetry(logging =>
                {
                    logging.IncludeScopes = true;
                    logging.IncludeFormattedMessage = true;

                    if (appInsightsConnStr is { Length: > 0 })
                    {
                        logging.AddAzureMonitorLogExporter(options =>
                        {
                            options.ConnectionString = appInsightsConnStr;

                            var appInsightsAuthStr = GetConfigurationValue(AppInsightsAuthenticationString, _config);
                            if (appInsightsAuthStr is { Length: > 0 })
                            {
                                var credOptions = AppInsightsCredentialOptions.ParseAuthenticationString(appInsightsAuthStr);
                                options.Credential = new ManagedIdentityCredential(credOptions.ClientId);
                            }
                        });
                    }

                    if (otlpEndpoint is { Length: > 0 })
                    {
                        logging.AddOtlpExporter();
                    }
                });
            });
        }

        private void ForceFlush(ILoggerFactory loggerFactory)
        {
            var serviceProvider = (IServiceProvider)loggerFactory.GetType()
            .GetField("_serviceProvider", BindingFlags.NonPublic | BindingFlags.Instance)
            .GetValue(loggerFactory);

            // Get all logger providers from the service provider
            var providers = serviceProvider?.GetServices<ILoggerProvider>() ?? Enumerable.Empty<ILoggerProvider>();

            foreach (var provider in providers)
            {
                if (provider is OpenTelemetryLoggerProvider otelProvider)
                {
                    otelProvider.Dispose();
                }
            }
        }

        private static string GetConfigurationValue(string key, IConfiguration configuration = null)
        {
            return configuration?[key] ?? Environment.GetEnvironmentVariable(key);
        }

        protected virtual void Dispose(bool disposing)
        {
            if (!_disposed)
            {
                if (disposing)
                {
                    _startupLoopTokenSource?.Dispose();
                    _hostStartSemaphore.Dispose();
                    DisposeRequestTrackingModule();
                }
                _disposed = true;
            }
        }

        public void Dispose() => Dispose(true);
    }
}